#!/usr/bin/python3
# -*- coding: utf-8 -*-
#author: 王树根
#date: 2016年3月15日 下午3:19:04
import re

from scrapy.selector.unified import Selector
from scrapy.spiders import Spider
from chinadaily.items import ChinaDailyItem

class ChinaDailySpider(Spider):
    name = "chinadaily"
    allowed_domains = "chinadaily.com.cn"
    start_urls = {
        "http://cn.chinadaily.com.cn"
    }
    
    def parse(self, response):
        html_sel = Selector(response)
        
        # 推荐新闻
        hot_news_nodes = html_sel.xpath(u'//div[@class="busBox1"]/div')
        news_id_re = re.compile(r'.*?/(\d{4}-\d{2}/\d{2})/content_(\d+).*')
        for hot_news_node in hot_news_nodes:
            item = ChinaDailyItem()
            item['image_url'] = hot_news_node.xpath(u'div[@class="mr10"]/a/img/@src').extract()[0]
            item['headline'] = hot_news_node.xpath(u'div/h3/a/text()').extract()[0]
            item['outline'] = hot_news_node.xpath(u'div/p/text()').extract()[0]
            item['link'] = hot_news_node.xpath(u'div[@class="mr10"]/a/@href').extract()[0]
            item['category'] = hot_news_node.xpath(u'div/div/span/a/text()').extract()[0]
            
            news_id_match_obj = news_id_re.match(item['link'])
            if not news_id_match_obj:
                continue
            item['news_id'] = news_id_re.match(item['link']).group(2)
            
            if not item['image_url'].startswith('http'):
                item['image_url'] = response.url + '/' + item['image_url']
        
            yield item